<?php
 mb_internal_encoding("utf-8");
 //gc_enabled();
 require "../document/class_corpus.php";
 require "../document/class_document.php";
 require "../document/class_document_xml.php";

 require "../files/tool_files_dirs.php";

 require "../misc/class_tag.php";
 require "../misc/tool_color.php";

 require "../tree/class_tree.php";
 require "../tree/class_tree_xhtml.php";
 require "../tree/class_structure.php";
 require "../tree/tool_tree.php";

 require "../array/tool_array.php";

 require "../pear_graph/tool_pear_graph.php";

 require "../maths/tool_maths.php";

 require "../string/tool_string.php";
 require "../string/class_segment.php";
 require "../string/class_suffixarray_new.php";
// require "../string/class_suffixarray_new.bak4.php";
 require "../string/suffix_array.php";

 require "./class_sqlite_document_new.php";

$dir="../xml/europa_complete_repair/";
//$dir="../xml/europa_test/";
//$dir="../xml/europa_test2/";
//$dir="../xml/test/";
$lg_candidates=array("cs","da","de","el","en","es","et","fi","fr","hu","it","lt","lv","nl","pl","pt","sl","sw");
$array_ids_europa=tool_corpus::get_unified_europa($dir,$lg_candidates);
//$motif_lg=implode("|",$lg_candidates);
//$motif_lg="fr";

foreach($lg_candidates as $motif_lg){

//$array_ids_europa=tool_corpus::europa_get_all_ids($dir);
$array_corpus_groupe=array();
$start=microtime(true);
$cpt=0;
$taille=0;
$array_bloc_str=array();
$factory_rstr=new suffixarray();
$array_bloc_doc=array();
//$factory_mot=new factory_mot();

$exp=new sqlite_experience("sqlite_document.$motif_lg.sdb");
$array_obj_sql=array(
  "document"=>new sqlite_document(),
  "bloc"=>new sqlite_bloc(),
  "rstr"=>new sqlite_rstr(),
  "graphie"=>new sqlite_graphie(),
);
 
$exp->writer_create_experience($array_obj_sql);
//die();
//*/
//$array_obj_sql=$exp->reader_experience();
//$exp->writer_drop_create_experience($array_obj_sql);

//print_r($array_obj_sql);
//die();
//$array_obj_sql=$experience->reader_experience();

foreach($array_ids_europa as $id_europa){
  $corpus=tool_corpus::get_corpus_regexp($dir,$id_europa."($motif_lg)");
  $array_doc=$corpus->__get_array_doc();
  foreach($array_doc as $document){
    $id_doc=mb_strcut($document->__get_id(),-6,2);
    $path=$document->__get_id();
    $structure=$document->__get_structure();
    $signature=$structure->__get_signature();
    tool_tree::cleanstructure($signature);
    $signature->__recalc_idn();

    $test=new RecursiveIteratorIterator($signature,RecursiveIteratorIterator::SELF_FIRST);
    $array_new_doc=array("path_doc"=>$path, "lg"=> $id_doc);
    $id_doc=$array_obj_sql["document"]->writer_values($array_new_doc);

    foreach($test as $id_bloc=>$bloc_xhtml){
      $i=array_push($array_bloc_str,tool_tree::tree_xhtml2texte($bloc_xhtml->__get_value()))-1;
      $array_new_bloc=array(
        "id_document"=>$id_doc,
        "contenu"=> $array_bloc_str[$i]
      );
      $id_bloc=$array_obj_sql["bloc"]->writer_values($array_new_bloc);
      $factory_rstr->__add_str($array_bloc_str[$i],$id_bloc);
//      $array_doc_bloc[$id_doc][]=$id_bloc;
//      $array_bloc_doc[$id_bloc]=$id_doc;
//      $factory_mot->get_all_mots($array_bloc_str[count($array_bloc_str)-1],$id_doc);
    }
  }
}

$all_res=$factory_rstr->__init();
$array_request=array();
$count=count($all_res);
echo $count;
echo "\n";
foreach($all_res as $c=>$rstr){
  $su=$factory_rstr->__get_suffix($rstr[1][0]);
  $global_str=$factory_rstr->__get_str($su[2]);
  $str=mb_substr($global_str,$su[0],$rstr[0]);
  $request_graphie=array(
    "graphie"=>$str
  );
  $id_graphie=$array_obj_sql["graphie"]->writer_values($request_graphie);

/*
  echo $count-$c;
  echo "-";
  echo count($rstr[1]);
  echo "\n";
*/
  foreach($rstr[1] as $pos){
    $su=$factory_rstr->__get_suffix($pos);
    $array_request[]=array(
      "id_bloc"=>$su[2]-1,
      "id_graphie"=>$id_graphie,
      "offset"=>$su[0],
      "len"=>$rstr[0]
    );
//    $array_obj_sql["rstr"]->writer_values($array_new_rstr);
  }
}
//print_r($array_request);
//die();
echo "\n";
echo count($array_request);
echo "\n";
$array_obj_sql["rstr"]->writer_array_values($array_request);
$end=microtime(true)-$start;
echo "\n total :: $end\n";
}
die();
die();

$zipf=array();

 /**********************
  * $rstr[0]==len
  * $rstr[1]==array_pos
  ***********************/
die();
foreach($all_res as $i=>$rstr){
  $su=$factory_rstr->__get_suffix($rstr[1][0]);
  $global_str=$factory_rstr->__get_str($su[2]);
  $str=mb_substr($global_str,$su[0],$rstr[0]);
  foreach($rstr[1] as $pos){
    $su=$factory_rstr->__get_suffix($pos);
    $id_doc=$array_bloc_doc[$su[2]];
    $zipf[$id_doc][$str]++;
  }
}

/*
 foreach($zipf as $id_doc=>$zipf_d){
   arsort($zipf_d);
   echo "id_doc : $id_doc**************************\n";
   foreach($zipf_d as $str=>$cpt){
     echo "[$cpt]\n    $str\n";
   }   
 }

die();
*/
function fill_graph(&$graph,$data_x,$data_y){
  fill_graph_aux($graph,$data_x,$data_y);
  return $graph->plot04();
}

function fill_graph_aux(&$graph,$data_x,$data_y){
  $graph->__add_array_x($data_x);
  $graph->__add_array_y($data_y);
}

$tool_graph=new Tool_pear_graph();
$list_ids_doc="";
foreach($zipf as $id_doc => $res_zipf){
 $list_ids_doc.="[$id_doc]";
 $zipf=tool_maths::zipf($res_zipf);
 $zipf_log=tool_maths::zipf_log($zipf);
 $x=array_keys($zipf_log);
 $y=array_values($zipf_log);
 $regression=tool_maths::regression_lineaire_simple($x,$y);

 $first_x=reset($x);
 $first_y=$regression["a"]*$first_x+$regression["b"];
 $last_x=end($x);
 $last_y=$regression["a"]*$last_x+$regression["b"];

 $tool_graph->__add_plot_title("coeff. $id_doc : $regression[r]");    
 fill_graph_aux($tool_graph,array($first_x,$last_x),array($first_y,$last_y));
// $tool_graph->__add_array_x(array($first_x,$last_x));
// $tool_graph->__add_array_y(array($first_y,$last_y));
}

$tool_graph->__add_title("regression linéaire rstr - zipf");
$graph=$tool_graph->line01();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./zipf_rstr_log_regression".$list_ids_doc.".svg",$graph_svg);
ob_clean();
die();

//$factory_mot->init_etiquetage();
//$start=microtime(true);
//$factory_mot->init_chunk();
//$end=microtime(true)-$start;
//echo "\n total :: $end";
//die();


//$repartition_chunk=$factory_mot->__get_repartiton_chunk();
$repartition_segment=$factory_mot->__get_repartition_segment();
//print_r($repartition_segment);
//die();
$res_all=$zipf=$res_type=array();
$res_type["p"]=$res_type["v"]=$res_type["i"]=0;
foreach($repartition_segment as $id_doc=>$array_bloc){
  foreach($array_bloc as $id_bloc=>$array_id_segment){
    foreach($array_id_segment as $id_segment){
      $mot=$factory_mot->__get_word($id_segment);
      $len=$mot->__get_len();
      $str=mb_strtolower($mot->__tostring());
      $type=$mot->__get_attribute("type");
      $id_doc=mb_strcut($id_doc,-6,2);
//      $res_all[$id_doc][$len][$str]++;
//      $res_all_type[$id_doc][$len][$str][$type]++;
//      $res_type[$id_doc][$type][$len][$str]++
//      $res_type[$id_doc][$type][$len]++;
//      if($type=="v") $res_v[$len]++;
//      if($type=="p") $res_p[$len]++;
//      if($type=="i") $res_i[$len]++;
//      $res_all[$id_doc][$len]++;
//      if($len>60) echo $mot."\n";
      $zipf[$id_doc][$str]++;
      if($type=="i") $zipf_i[$id_doc][$str]++;
      if($type=="v") $zipf_v[$id_doc][$str]++;
      if($type=="p") $zipf_p[$id_doc][$str]++;
    }
  }
}
/*
function fill_graph(&$graph,$data_x,$data_y){
  fill_graph_aux($graph,$data_x,$data_y);
  return $graph->plot04();
}

function fill_graph_aux(&$graph,$data_x,$data_y){
  $graph->__add_array_x($data_x);
  $graph->__add_array_y($data_y);
}
*/
$tool_graph=new Tool_pear_graph();
$list_ids_doc="";
foreach($zipf as $id_doc => $res_zipf){
 $list_ids_doc.="[$id_doc]";
 $zipf=tool_maths::zipf($res_zipf);
 $zipf_log=tool_maths::zipf_log($zipf);
 $x=array_keys($zipf_log);
 $y=array_values($zipf_log);
 $regression=tool_maths::regression_lineaire_simple($x,$y);

 $first_x=reset($x);
 $first_y=$regression["a"]*$first_x+$regression["b"];
 $last_x=end($x);
 $last_y=$regression["a"]*$last_x+$regression["b"];

 $tool_graph->__add_plot_title("coeff. $id_doc : $regression[r]");    
 fill_graph_aux($tool_graph,array($first_x,$last_x),array($first_y,$last_y));
// $tool_graph->__add_array_x(array($first_x,$last_x));
// $tool_graph->__add_array_y(array($first_y,$last_y));
}

$tool_graph->__add_title("regression linéaire - zipf");
$graph=$tool_graph->line01();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./svg_zipf/zipf_log_regression".$list_ids_doc.".svg",$graph_svg);
ob_clean();
die();

$tool_graph=new Tool_pear_graph();
$tool_graph->__add_title("rang - effectif");
foreach($zipf as $id_doc => $res_zipf){
  $res_zipf=tool_maths::own_zipf_etiquette($res_zipf,array("i"=>$zipf_i[$id_doc],"v"=>$zipf_v[$id_doc],"p"=>$zipf_p[$id_doc]));
  $zipf_log=tool_maths::zipf_log2($res_zipf[0]);
  $tool_graph->__add_plot_title("zipf : $id_doc");    
  fill_graph_aux($tool_graph,$zipf_log[0],$zipf_log[1]);
  $graph=$tool_graph->plot04();
  ob_start();
  $graph->done();
  $graph_svg=ob_get_contents();
  Tool_files::file_write("./svg_zipf/zipf.svg",$graph_svg);
  ob_clean();
}
die();
$graph=$tool_graph->plot04();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./svg_zipf/zipf_i.svg",$graph_svg);
ob_clean();
die();
/*
$tool_graph=new Tool_pear_graph();
foreach($zipf as $id_doc => $res_zipf){
  $res_zipf=tool_maths::own_zipf_etiquette($res_zipf,array("i"=>$zipf_i[$id_doc],"v"=>$zipf_v[$id_doc],"p"=>$zipf_p[$id_doc]));
  $zipf_log=tool_maths::zipf_log($res_zipf[0]);
  $tool_graph->__add_array_x(array_keys($zipf_log));
  $tool_graph->__add_array_y(array_values($zipf_log));
  $tool_graph->__add_plot_title("zipf : $id_doc");

  $zipf_log_v=tool_maths::zipf_log($res_zipf[1]["v"]);
  $tool_graph->__add_array_x(array_keys($zipf_log_v));
  $tool_graph->__add_array_y(array_values($zipf_log_v));
  $tool_graph->__add_plot_title("zipf v : $id_doc");
}

$tool_graph->__add_title("rang - longueur");
$graph=$tool_graph->plot04();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./svg_zipf/zipf_v.svg",$graph_svg);
ob_clean();

$tool_graph=new Tool_pear_graph();
foreach($zipf as $id_doc => $res_zipf){
  $res_zipf=tool_maths::own_zipf_etiquette($res_zipf,array("i"=>$zipf_i[$id_doc],"v"=>$zipf_v[$id_doc],"p"=>$zipf_p[$id_doc]));
  $zipf_log=tool_maths::zipf_log($res_zipf[0]);
  $tool_graph->__add_array_x(array_keys($zipf_log));
  $tool_graph->__add_array_y(array_values($zipf_log));
  $tool_graph->__add_plot_title("zipf : $id_doc");

  $zipf_log_p=tool_maths::zipf_log($res_zipf[1]["p"]);
  $tool_graph->__add_array_x(array_keys($zipf_log_p));
  $tool_graph->__add_array_y(array_values($zipf_log_p));
  $tool_graph->__add_plot_title("zipf p : $id_doc");
}

$tool_graph->__add_title("rang - effectif");
$graph=$tool_graph->plot04();
ob_start();
$graph->done();
$graph_svg=ob_get_contents();
Tool_files::file_write("./svg_zipf/zipf_p.svg",$graph_svg);
ob_clean();
*/
/*
foreach($res_all as $id_doc => $array_len_str){
  ksort($array_len_str);
  $array_res=array();
  foreach($array_len_str as $len=>$array_str){
    asort($array_str);
    foreach($array_str as $str=>$nb_occ){
      $array_res[]=$nb_occ;
    }
  }
  $tool_graph->__add_array_x(array_keys($array_res));
  $tool_graph->__add_array_y(array_values($array_res));
  $tool_graph->__add_plot_title("len : $id_doc");
}
*/
/*
foreach($res_all as $id_doc => $len_word){
  ksort($len_word);
  $tool_graph->__add_array_x(array_keys($len_word));
  $tool_graph->__add_array_y(array_values($len_word));
  $tool_graph->__add_plot_title("len_word : $id_doc");
  foreach($res_type[$id_doc] as $type=>$array_res_type){
    ksort($array_res_type);
    $tool_graph->__add_array_x(array_keys($array_res_type));
    $tool_graph->__add_array_y(array_values($array_res_type));
    $tool_graph->__add_plot_title("type : $type - $id_doc");
  }
}
*/
//die();

/*
$len_chunk=array();
$repartition_chunk=$factory_mot->__get_repartition_chunk();
//print_r($repartition_chunk);
//die();
foreach($repartition_chunk as $id_doc=>$array_bloc){
  $id_doc=mb_strcut($id_doc,-6,2);
  foreach($array_bloc as $id_bloc=>$array_chunk){
    foreach($array_chunk as $array_id_segment){
      $len=0; 
      foreach($array_id_segment as $id_segment){
        $mot=$factory_mot->__get_word($id_segment);
        //if($mot->__get_attribute("type")!="v")
          $len+=$mot->__get_len();
      }
      $len_chunk[$id_doc][$len]++;
    }
  }
}
*/
//die();


/*
foreach($len_chunk as $id_doc => $len){
//  if($id_doc=="fr"){
    ksort($len);
    $tool_graph->__add_array_x(array_keys($len));
    $tool_graph->__add_array_y(array_values($len));
    $tool_graph->__add_plot_title("len_chunk : $id_doc");
//  }
}
*/
//die();
//print_r($zipf_log);
//die();
//print_r($zipf);
//die();
//$tool_graph->__add_array_x(array_keys($res_all));
//$tool_graph->__add_array_y(array_values($res_all));
//$tool_graph->__add_array_x(array_keys($zipf));
//$tool_graph->__add_array_y(array_values($zipf));
//$tool_graph->__add_array_x(array_keys($zipf_log_fi));
//$tool_graph->__add_array_y(array_values($zipf_log_fi));
//$tool_graph->__add_plot_title("zipf finnois log*log");
//$tool_graph->__add_plot_title("zipf");


/*
$tool_graph->__add_array_x(array_keys($res_v));
$tool_graph->__add_array_x(array_keys($res_i));
$tool_graph->__add_array_x(array_keys($res_p));
*/
/*
$tool_graph->__add_array_y(array_values($res_v));
$tool_graph->__add_array_y(array_values($res_i));
$tool_graph->__add_array_y(array_values($res_p));
*/
//$tool_graph->__add_title("droite de régression sur le rapport longueur*effectif de la loi de zipf des documents");
//$tool_graph->__add_title("rapport longueur*effectif de la loi de zipf des documents");
?>
